import time

import scrapy
from datetime import datetime
from urllib.parse import urlencode
import json
from web_crawler.config import get_headers,get_cookies
from ..items import KlineItem
from ..utils import MySQLPool

class XueqiuKlinesSpider(scrapy.Spider):
    name = 'xueqiu_klines'
    allowed_domains = ["stock.xueqiu.com"]
    custom_settings = {
        'CONCURRENT_REQUESTS': 64,  # 可以在这里覆盖全局设置
        'CONCURRENT_REQUESTS_PER_DOMAIN': 20,
        'DOWNLOAD_DELAY': 0.1,
        'ITEM_PIPELINES': {
            'web_crawler.pipeline.klines.XueqiuKlinePipeline': 300,
        }
    }
    COUNT = "-2"
    # 从MySQL获取symbol列表的SQL
    SYMBOL_SQL = """
       SELECT distinct symbol FROM xueqiu_stocks 
       """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.pool = MySQLPool()
        self.symbols = self._fetch_symbols_from_db() # 股票列表


    def _fetch_symbols_from_db(self):
        """从MySQL获取symbol列表"""
        conn = self.pool.get_conn()

        try:
            with conn.cursor() as cursor:
                cursor.execute(self.SYMBOL_SQL)
                return [row['symbol'] for row in cursor.fetchall()]
        finally:
            conn.close()

    def start_requests(self):
        """为每个symbol生成K线请求"""
        for symbol in self.symbols:
            params = {
                "symbol": symbol,
                "begin": str(int(time.time() * 1000)),
                "period": "day",
                "type": "before",
                "count": self.COUNT,  # 只获取5条数据用于测试
                "indicator": "kline,pe,pb,ps,pcf,market_capital,macd,kdj,boll"
            }
            yield scrapy.Request(
                url=f"https://stock.xueqiu.com/v5/stock/chart/kline.json?{urlencode(params)}",
                headers=get_headers(),
                cookies=get_cookies(),
                callback=self.parse_kline,
                meta={'symbol': symbol}
            )


    def parse_kline(self, response):
        """解析K线数据"""
        try:
            data = json.loads(response.text)
            symbol = response.meta['symbol']

            # 验证接口返回状态
            if data.get('error_code') != 0:
                self.logger.error(f"获取{symbol}K线数据失败: {data.get('error_description')}")
                return
            # 验证数据结构
            if not data.get('data') or 'item' not in data['data']:
                self.logger.error(f"无效数据结构: {data}")
                return

            column_mapping = {
                'timestamp': 'timestamp',
                'volume': 'volume',
                'open': 'open_price',
                'close': 'close_price',
                'high': 'high_price',
                'low': 'low_price',
                'chg': 'price_change',
                'percent': 'percent_change',
                'turnoverrate': 'turnover_rate',
                'amount': 'amount',
                'dea': 'macd_dea',
                'dif': 'macd_dif',
                'macd': 'macd',
                'ub': 'boll_upper',
                'lb': 'boll_lower',
                'ma20': 'ma20',
                'kdjk': 'kdj_k',
                'kdjd': 'kdj_d',
                'kdjj': 'kdj_j',
                'pe': 'pe_ttm',
                'pb': 'pb',
                'ps': 'ps',
                'pcf': 'pcf',
                'market_capital': 'market_cap'
            }

            # 解析每条K线数据
            for item_values in data['data']['item']:
                # 创建KlineItem对象
                kline_item = KlineItem()
                # 设置基本字段
                kline_item['symbol'] = symbol
                # 动态映射字段
                for col, val in zip(data['data']['column'], item_values):
                    if col in column_mapping:
                        field_name = column_mapping[col]
                        kline_item[field_name] = val

                # 处理时间字段
                if 'timestamp' in kline_item and kline_item['timestamp']:
                    timestamp = kline_item['timestamp']
                    kline_item['trade_date'] = datetime.fromtimestamp(timestamp / 1000).strftime('%Y-%m-%d')
                yield kline_item

        except Exception as e:
            self.logger.error(f"解析K线数据出错: {e}")